package com.lzx.user.controller.Crawler;

import com.lzx.common.utils.R;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import java.util.ArrayList;
import java.util.List;

/**
 * @author 程序员星星
 * @date 2023/3/15
 * @Description
 */
@RestController
@RequestMapping("/user/crawler/web")
public class CrawlerController {

    private static final String URL_TEMPLATE = "https://blog.csdn.net/%s/article/list/%d";
    private static final String USER_NAME = "crazy1013";
    private static final int PAGE_NUM = 1;

    //java爬取csdn文章
    @PostMapping("/crawlerCSDN/{username}")
    public R crawlerCSDN(@PathVariable String username) throws Exception {
        String url = String.format(URL_TEMPLATE, username, PAGE_NUM);
        Document doc = Jsoup.connect(url).get();
        Elements articleElements = doc.select("div.article-list div.article-item-box h4 a");
        List<Article> articles = new ArrayList<>();
        for (Element articleElement : articleElements) {
            String articleUrl = articleElement.attr("href");
            Document articleDoc = Jsoup.connect(articleUrl).get();
            Element articleTitleElement = articleDoc.selectFirst("h1.title-article");
            String articleTitle = articleTitleElement.text();
            Element articleBodyElement = articleDoc.selectFirst("div#article_content");
            String articleBody = articleBodyElement.html();
            Article article = new Article();
            article.setTitle(articleTitle);
            article.setContent(articleBody);
            articles.add(article);
            // TODO: 处理文章内容
//            System.out.println(articleTitle);
//            System.out.println(articleBody);
        }
        return R.ok().data("list", articles);
    }

}
